# -*- coding: utf-8 -*-
import scrapy
import json
from scrapy.http.response.html import HtmlResponse
from ..items import ItemType,JiayuanUserInfo
from DoubanBook.common.redis_opt import RedisOpt,RedisConstant

class JiayuanSpider(scrapy.Spider):
    name = 'jiayuan'
    allowed_domains = ['jiayuan.com']
    start_urls = ['https://search.jiayuan.com/v2/search_v2.php?sex=f&key=&sn=default&sv=1&p=1000001&f=&listStyle=bigPhoto&pri_uid=0&jsversion=v5']

    pageIndex = 1
    redisOpt = RedisOpt.get_instance()

    #爬取列表分页
    def parse(self, response:HtmlResponse):
        str_body = response.body
        if isinstance(response.body, bytes):
            str_body = bytes.decode(response.body).replace('##jiayser##','',3).replace('//','',3)
        resjson = json.loads(str_body)

        info = JiayuanUserInfo()
        for item in resjson['userInfo']:
            info['itemName'] = ItemType.JIAY_USER.value
            info['uid'] = item['uid']
            info['nickName'] = item['nickname']
            info['sex'] = item['sexValue']
            info['age'] = item['age']
            info['marriage'] = item['marriage']
            info['height'] = item['height']
            info['education'] = item['education']
            self.logger.info('item image>>>{}'.format(item['image']))
            info['image'] = item['image']
            info['workLocation'] = item['work_location']

            if 'xjhykj_f.jpg' not in info['image']:
                info['realUid'] = item['realUid']
                yield scrapy.Request(url='https://www.jiayuan.com/{}?fxly=search_v2'.format(info['realUid']), callback=self.parse_detail,dont_filter=True,meta={'item':info})

        pageTotal = 1#resjson['pageTotal']
        if(self.pageIndex < pageTotal):
            yield scrapy.Request(url='https://search.jiayuan.com/v2/search_v2.php?sex=f&key=&sn=default&sv=1&p={}&f=&listStyle=bigPhoto&pri_uid=0&jsversion=v5'.format(self.pageIndex),
                                 callback=self.parse,dont_filter=True)
            self.pageIndex = self.pageIndex + 1
        else:
            print('分页爬虫完成')
        pass


    # 爬取详情页 这里会有ip访问限制
    def parse_detail(self, response:HtmlResponse):
        # https://www.jiayuan.com/228990603?fxly=search_v2
        userInfo = JiayuanUserInfo()
        userInfo['itemName'] = ItemType.JIAY_USER.value
        if 'captcha.php?back_url' not in '{}'.format(response.url):
            str_body = response.body
            metaItem = response.meta['item']
            self.logger.info('parse_detail response.url>>>{}'.format(response.url))

            menber_info_ele = response.xpath('//div[contains(@class, "member_info_r")]')
            self.logger.info('realUid>>>>{}'.format(metaItem['realUid']))
            if metaItem['realUid'] is not None: #ID:
                userInfo['realUid'] = metaItem['realUid']
                userInfo['uid'] = metaItem['uid']
                userInfo['nickName'] = metaItem['nickName']
                userInfo['charm'] = menber_info_ele.xpath('./div[@class="ml_ico"]//h6/text()').extract_first()
                userInfo['menberType'] = menber_info_ele.xpath('.//p[@class="member_ico_box"]/span[@class="member_dj"]/text()').extract_first()
                userInfo['age_marriage_from'] = menber_info_ele.xpath('.//div[@class="member_name"]/text()').extract()

                menber_infos_ele = menber_info_ele.xpath('.//div[@class="member_info_list"]/li')
                #for menber_info_item in menber_infos_ele:
                for index in iter(menber_infos_ele):
                    item = menber_infos_ele[index]
                    value = item.xpath('./ul[@class="fl"]/text()').extract()
                    if 0 == index:#学历
                        userInfo['education'] = value
                    elif 1 == index:#身高
                        userInfo['height'] = value
                    elif 2 == index:#购车
                        userInfo['carType'] = value
                    elif 5 == index:#体重
                        userInfo['weight'] = value
                    elif 6 == index:#星座
                        userInfo['constellation'] = value
                    elif 7 == index:#民族
                        userInfo['nation'] = value
                    elif 8 == index:  # 属相
                        userInfo['yearbirth'] = value
                    elif 9 == index:  # 血型
                        userInfo['bloodType'] = value
                    else:
                        self.logger.warn('未定义的属性 >>>{}  >>> {}'.format(index,value))
                img_list = []
                img_list_ele = response.xpath('.//div[@id="bigImg"]//img')
                for item in iter(img_list_ele):
                    imgsrc = item.xpath('./@_src').extract_first()
                    self.logger.info('imgsrc>>>>{}'.format(imgsrc))
                    img_list.append(imgsrc)
                if len(img_list) > 0:
                    self.logger.info('img_list>>>>{}'.format(img_list))
                    userInfo['images'] = '#'.join(img_list)#  以#分割多个图片
                yield userInfo
            else:
                self.logger.error('uid为空  response.url>>>>{}'.format(response.url))
        else:
            self.redisOpt.s_set(RedisConstant.JIAYUAN_PORXY_STATUS, '0')
            self.logger.info('触发反爬虫')
    pass

